home *** CD-ROM | disk | FTP | other *** search
- #! /usr/bin/python
- # -*- coding: UTF-8 -*- vim: et ts=4 sw=4
-
- # Copyright © 2010 Piotr Ożarowski <piotr@debian.org>
- #
- # Permission is hereby granted, free of charge, to any person obtaining a copy
- # of this software and associated documentation files (the "Software"), to deal
- # in the Software without restriction, including without limitation the rights
- # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- # copies of the Software, and to permit persons to whom the Software is
- # furnished to do so, subject to the following conditions:
- #
- # The above copyright notice and this permission notice shall be included in
- # all copies or substantial portions of the Software.
- #
- # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- # THE SOFTWARE.
-
- from __future__ import with_statement
- import logging
- import os
- import re
- import sys
- from filecmp import dircmp, cmpfiles
- from optparse import OptionParser, SUPPRESS_HELP
- from os.path import isdir, islink, exists, join
- from shutil import rmtree, copy as fcopy
- from stat import ST_MODE, S_IXUSR, S_IXGRP, S_IXOTH
- sys.path.insert(1, '/usr/share/python/')
- from debpython.debhelper import DebHelper
- from debpython.depends import Dependencies
- from debpython.version import SUPPORTED, DEFAULT, \
- debsorted, getver, vrepr, parse_pycentral_vrange, \
- get_requested_versions, parse_vrange, vrange_str
- from debpython.pydist import validate as validate_pydist, \
- PUBLIC_DIR_RE
- from debpython.tools import sitedir, relative_symlink, \
- shebang2pyver, clean_egg_name
- from debpython.option import Option
-
- # initialize script
- logging.basicConfig(format='%(levelname).1s: %(module)s:%(lineno)d: '
- '%(message)s')
- log = logging.getLogger(__name__)
- os.umask(022)
- EGGnPTH_RE = re.compile(r'(.*?)(-py\d\.\d+)?(.*?)(\.egg-info|\.pth)$')
-
- """TODO: move it to manpage
- Examples:
- dh_python2
- dh_python2 -V 2.4- # public files only, Python >= 2.4
- dh_python2 -p python-foo -X 'bar.*' /usr/lib/baz/ # private files in
- python-foo package
- """
-
- # naming conventions used in the file:
- # * version - tuple of integers
- # * ver - string representation of version
- # * vrange - version range, pair of max and min versions
- # * fn - file name (without path)
- # * fpath - file path
-
-
- ### FILES ######################################################
- def fix_locations(package):
- """Move files to the right location."""
- found_versions = {}
- for version in SUPPORTED:
- ver = vrepr(version)
- to_check = [i % ver for i in (\
- 'usr/local/lib/python%s/site-packages',
- 'usr/local/lib/python%s/dist-packages',
- 'var/lib/python-support/python%s',
- 'usr/lib/pymodules/python%s')]
- if version >= (2, 6):
- to_check.append("usr/lib/python%s/site-packages" % ver)
- dstdir = sitedir(version, package)
-
- for location in to_check:
- srcdir = "debian/%s/%s" % (package, location)
- if isdir(srcdir):
- if ver in found_versions:
- log.error('files for version %s '
- 'found in two locations:\n %s\n %s',
- ver, location, found_versions[ver])
- exit(2)
- log.warn('Python %s should install files in %s. '
- 'Did you forget "--install-layout=deb"?',
- ver, sitedir(version))
- if not isdir(dstdir):
- os.makedirs(dstdir)
- # TODO: what about relative symlinks?
- log.debug('moving files from %s to %s', srcdir, dstdir)
- os.renames(srcdir, dstdir)
- found_versions[ver] = location
-
- # do the same with debug locations
- dbg_to_check = ['usr/lib/debug/%s' % i for i in to_check]
- dbg_to_check.append("usr/lib/debug/usr/lib/pyshared/python%s" % ver)
- dstdir = sitedir(version, package, gdb=True)
-
- for location in dbg_to_check:
- srcdir = "debian/%s/%s" % (package, location)
- if isdir(srcdir):
- if not isdir(dstdir):
- os.makedirs(dstdir)
- log.debug('moving files from %s to %s', srcdir, dstdir)
- os.renames(srcdir, dstdir)
-
-
- ### SHARING FILES ##############################################
- def share(package, stats, options):
- """Move files to /usr/share/pyshared/ if possible."""
- if package.endswith('-dbg'):
- # nothing to share in debug packages
- return
- pubvers = debsorted(i for i in stats['public_vers'] if i[0] == 2)
- if len(pubvers) > 1:
- for pos, version1 in enumerate(pubvers):
- dir1 = sitedir(version1, package)
- for version2 in pubvers[pos + 1:]:
- dir2 = sitedir(version2, package)
- dc = dircmp(dir1, dir2)
- share_2x(dir1, dir2, dc)
- elif len(pubvers) == 1:
- # TODO: remove this once file conflicts will not be needed anymore
- move_to_pyshared(sitedir(pubvers[0], package))
-
- for version in stats['public_ext']:
- create_ext_links(sitedir(version, package))
-
- if options.guess_versions and pubvers:
- versions = get_requested_versions(options.vrange)
- for version in (i for i in versions if i[0] == 2):
- if version not in pubvers:
- log.debug('guessing files for Python %s', vrepr(version))
- versions_without_ext = debsorted(set(pubvers) -\
- stats['public_ext'])
- if not versions_without_ext:
- log.error('you most probably have to build extension '
- 'for python%s.', vrepr(version))
- exit(12)
- srcver = versions_without_ext[0]
- if srcver in stats['public_vers']:
- stats['public_vers'].add(version)
- share_2x(sitedir(srcver, package), sitedir(version, package))
-
-
- def move_to_pyshared(dir1):
- # dir1 starts with debian/packagename/usr/lib/pythonX.Y/*-packages/
- debian, package, path = dir1.split('/', 2)
- dstdir = join(debian, package, 'usr/share/pyshared/', \
- '/'.join(dir1.split('/')[6:]))
-
- fext = lambda fname: fname.rsplit('.', 1)[-1]
-
- for i in os.listdir(dir1):
- fpath1 = join(dir1, i)
- if isdir(fpath1):
- if any(fn for fn in os.listdir(fpath1) if fext(fn) != 'so'):
- # at least one file that is not an extension
- move_to_pyshared(join(dir1, i))
- else:
- if fext(i) == 'so':
- continue
- fpath2 = join(dstdir, i)
- if not exists(fpath2):
- if not exists(dstdir):
- os.makedirs(dstdir)
- os.rename(fpath1, fpath2)
- relative_symlink(fpath2, fpath1)
-
-
- def create_ext_links(dir1):
- """Create extension symlinks in /usr/lib/pyshared/pythonX.Y.
-
- These symlinks are used to let dpkg detect file conflicts with
- python-support and python-central packages.
- """
-
- debian, package, path = dir1.split('/', 2)
- python, _, module_subpath = path[8:].split('/', 2)
- dstdir = join(debian, package, 'usr/lib/pyshared/', python, module_subpath)
-
- for i in os.listdir(dir1):
- fpath1 = join(dir1, i)
- if isdir(fpath1):
- create_ext_links(fpath1)
- elif i.rsplit('.', 1)[-1] == 'so':
- fpath2 = join(dstdir, i)
- if exists(fpath2):
- continue
- if not exists(dstdir):
- os.makedirs(dstdir)
- relative_symlink(fpath1, join(dstdir, i))
-
-
- def share_2x(dir1, dir2, dc=None):
- """Move common files to pyshared and create symlinks in original
- locations."""
- debian, package, path = dir2.split('/', 2)
- # dir1 starts with debian/packagename/usr/lib/pythonX.Y/*-packages/
- dstdir = join(debian, package, 'usr/share/pyshared/', \
- '/'.join(dir1.split('/')[6:]))
- if not exists(dstdir):
- os.makedirs(dstdir)
- if dc is None: # guess/copy mode
- if not exists(dir2):
- os.makedirs(dir2)
- common_dirs = []
- common_files = []
- for i in os.listdir(dir1):
- if isdir(join(dir1, i)):
- common_dirs.append([i, None])
- else:
- # directories with .so files will be blocked earlier
- common_files.append(i)
- else:
- common_dirs = dc.subdirs.iteritems()
- common_files = dc.common_files
- # dircmp returns common names only, lets check files more carefully...
- common_files = cmpfiles(dir1, dir2, common_files, shallow=False)[0]
-
- for fn in common_files:
- if fn.endswith('.so'):
- # in unlikely case where extensions are exactly the same
- continue
- fpath1 = join(dir1, fn)
- fpath2 = join(dir2, fn)
- fpath3 = join(dstdir, fn)
- # do not touch symlinks created by previous loop or other tools
- if dc and not islink(fpath1):
- # replace with a link to pyshared
- os.rename(fpath1, fpath3)
- relative_symlink(fpath3, fpath1)
- if dc is None: # guess/copy mode
- if islink(fpath1):
- # ralative links will work as well, it's always the same level
- os.symlink(os.readlink(fpath1), fpath2)
- else:
- if exists(fpath3):
- # cannot share it, pyshared contains another copy
- fcopy(fpath1, fpath2)
- else:
- # replace with a link to pyshared
- os.rename(fpath1, fpath3)
- relative_symlink(fpath3, fpath1)
- relative_symlink(fpath3, fpath2)
- else:
- os.remove(fpath2)
- relative_symlink(fpath3, fpath2)
- for dn, dc in common_dirs:
- share_2x(join(dir1, dn), join(dir2, dn), dc)
-
-
- ### PACKAGE DETAILS ############################################
- def scan(package, dname=None):
- """Gather statistics about Python files in given package."""
- r = {'requires.txt': set(),
- 'shebangs': set(),
- 'public_vers': set(),
- 'private_dirs': {},
- 'compile': False,
- 'public_ext': set()}
-
- dbg_package = package.endswith('-dbg')
-
- if not dname:
- proot = "debian/%s" % package
- if dname is False:
- private_to_check = []
- else:
- private_to_check = [i % package for i in
- ('usr/lib/%s', 'usr/lib/games/%s',
- 'usr/share/%s', 'usr/share/games/%s')]
- else:
- proot = join('debian', package, dname.strip('/'))
- private_to_check = [dname[1:]]
-
- for root, dirs, file_names in os.walk(proot):
- # ignore Python 3.X locations
- if '/usr/lib/python3' in root or\
- '/usr/local/lib/python3' in root:
- # warn only once
- warn = root[root.find('/lib/python'):].count('/') == 2
- if warn:
- log.warning('Python 3.x location detected, '
- 'please use dh_python3: %s', root)
- continue
-
- bin_dir = private_dir = None
- public_dir = PUBLIC_DIR_RE.match(root)
- if public_dir:
- version = getver(public_dir.group(1))
- if root.endswith('-packages'):
- r['public_vers'].add(version)
- else:
- version = False
- for i in private_to_check:
- if root.startswith(join('debian', package, i)):
- private_dir = '/' + i
- break
- else: # i.e. not public_dir and not private_dir
- if len(root.split('/', 6)) < 6 and (\
- root.endswith('/sbin') or root.endswith('/bin') or\
- root.endswith('/usr/games')):
- # /(s)bin or /usr/(s)bin or /usr/games
- bin_dir = root
-
- # handle some EGG related data (.egg-info dirs)
- for name in dirs:
- if name.endswith('.egg-info'):
- if dbg_package:
- rmtree(join(root, name))
- dirs.pop(dirs.index(name))
- continue
- clean_name = clean_egg_name(name)
- if clean_name != name:
- log.info('renaming %s to %s', name, clean_name)
- os.rename(join(root, name), join(root, clean_name))
- if root.endswith('.egg-info') and 'requires.txt' in file_names:
- r['requires.txt'].add(join(root, 'requires.txt'))
- continue
-
- # check files
- for fn in file_names:
- fext = fn.rsplit('.', 1)[-1]
- if fext in ('pyc', 'pyo'):
- os.remove(join(root, fn))
- continue
- if public_dir:
- if dbg_package and fext not in ('so', 'h'):
- os.remove(join(root, fn))
- continue
-
- elif private_dir:
- mode = os.stat(join(root, fn))[ST_MODE]
- if mode is S_IXUSR or mode is S_IXGRP or mode is S_IXOTH:
- res = shebang2pyver(join(root, fn))
- if res:
- r['private_dirs'].setdefault(private_dir, {})\
- .setdefault('shebangs', set()).add(res)
-
- if public_dir or private_dir:
- if fext == 'so':
- (r if public_dir else
- r['private_dirs'].setdefault(private_dir, {}))\
- .setdefault('public_ext', set()).add(version)
- continue
- elif fext == 'py':
- (r if public_dir else
- r['private_dirs'].setdefault(private_dir, {}))\
- ['compile'] = True
- continue
-
- # .egg-info files
- if fn.endswith('.egg-info'):
- clean_name = clean_egg_name(fn)
- if clean_name != fn:
- log.info('renaming %s to %s', fn, clean_name)
- os.rename(join(root, fn), join(root, clean_name))
- continue
- # search for scripts in bin dirs
- if bin_dir:
- fpath = join(root, fn)
- res = shebang2pyver(fpath)
- if res:
- r['shebangs'].add(res)
-
- if dbg_package:
- # remove empty directories in -dbg packages
- proot = proot + '/usr/lib'
- for root, dirs, file_names in os.walk(proot, topdown=False):
- if '-packages/' in root and not file_names:
- try:
- os.rmdir(root)
- except:
- pass
-
- log.debug("package %s details = %s", package, r)
- return r
-
-
- ################################################################
- def main():
- usage = '%prog -p PACKAGE [-V [X.Y][-][A.B]] DIR_OR_FILE [-X REGEXPR]\n'
- parser = OptionParser(usage, version='%prog 2.0~beta1',
- option_class=Option)
- parser.add_option('--no-guessing-versions', action='store_false',
- dest='guess_versions', default=True,
- help='disable guessing other supported Python versions')
- parser.add_option('--no-guessing-deps', action='store_false',
- dest='guess_deps', default=True, help='disable guessing dependencies')
- parser.add_option('--skip-private', action='store_true',
- dest='skip_private', default=False,
- help='don\'t check private directories')
- parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
- default=False, help='turn verbose mode on')
- # arch=False->arch:all only, arch=True->arch:any only, None->all of them
- parser.add_option('-i', '--indep', action='store_false',
- dest='arch', default=None,
- help='act on architecture independent packages')
- parser.add_option('-a', '--arch', action='store_true',
- dest='arch', help='act on architecture dependent packages')
- parser.add_option('-q', '--quiet', action='store_false', dest='verbose',
- help='be quiet')
- parser.add_option('-p', '--package', action='append', dest='package',
- help='act on the package named PACKAGE')
- parser.add_option('-N', '--no-package', action='append', dest='no_package',
- help='do not act on the specified package')
- parser.add_option('-V', type='version_range', dest='vrange',
- help='specify list of supported Python versions. ' +\
- 'See pycompile(1) for examples')
- parser.add_option('-X', '--exclude', action='append', dest='regexpr',
- help='exclude items that match given REGEXPR. You may use this option '
- 'multiple times to build up a list of things to exclude.')
- parser.add_option('--depends', action='append', dest='depends',
- help='translate given requirements into Debian dependencies '
- 'and add them to ${python:Depends}. '
- 'Use it for missing items in requires.txt.')
- parser.add_option('--recommends', action='append', dest='recommends',
- help='translate given requirements into Debian '
- 'dependencies and add them to ${python:Recommends}')
- parser.add_option('--suggests', action='append', dest='suggests',
- help='translate given requirements into Debian '
- 'dependencies and add them to ${python:Suggests}')
- # ignore some debhelper options:
- parser.add_option('-O', help=SUPPRESS_HELP)
-
- (options, args) = parser.parse_args()
- # regexpr option type is not used so lets check patterns here
- for pattern in options.regexpr or []:
- # fail now rather than at runtime
- try:
- pattern = re.compile(pattern)
- except:
- log.error('regular expression is not valid: %s', pattern)
- exit(1)
-
- if not options.vrange and exists('debian/pyversions'):
- log.debug('parsing version range from debian/pyversions')
- with open('debian/pyversions') as fp:
- for line in fp:
- line = line.strip()
- if line and not line.startswith('#'):
- options.vrange = parse_vrange(line)
- break
-
- # disable PyDist if dh_pydeb is used
- if options.guess_deps:
- try:
- fp = open('debian/rules', 'r')
- except IOError:
- log.warning('cannot open debian/rules file')
- else:
- if re.compile('\n\s*dh_pydeb').search(fp.read()):
- log.warning('dh_pydeb detected, PyDist feature disabled')
- options.guess_deps = False
-
- private_dir = None if not args else args[0]
- # TODO: support more than one private dir at the same time (see :meth:scan)
- if options.skip_private:
- private_dir = False
-
- if options.verbose or os.environ.get('DH_VERBOSE') == '1':
- log.setLevel(logging.DEBUG)
- log.debug('argv: %s', sys.argv)
- log.debug('options: %s', options)
- log.debug('args: %s', args)
-
- dh = DebHelper(options.package, options.no_package)
- if not options.vrange and dh.python_version:
- options.vrange = parse_pycentral_vrange(dh.python_version)
-
- for package, pdetails in dh.packages.iteritems():
- if options.arch is False and pdetails['arch'] != 'all' or \
- options.arch is True and pdetails['arch'] == 'all':
- continue
- log.debug('processing package %s...', package)
- fix_locations(package)
- stats = scan(package, private_dir)
- share(package, stats, options)
-
- dependencies = Dependencies(package,
- dh.packages[package]['uses_breaks'])
- dependencies.parse(stats, options)
- dependencies.export_to(dh)
-
- if stats['public_vers']:
- dh.addsubstvar(package, 'python:Versions', \
- ', '.join(sorted(vrepr(stats['public_vers']))))
- ps = package.split('-', 1)
- if len(ps) > 1 and ps[0] == 'python':
- dh.addsubstvar(package, 'python:Provides', \
- ', '.join("python%s-%s" % (i, ps[1])\
- for i in sorted(vrepr(stats['public_vers']))))
-
- pyclean_added = False # invoke pyclean only once in maintainer script
- if stats['compile']:
- dh.autoscript(package, 'preinst', 'preinst-pycentral-clean', '')
- dh.autoscript(package, 'postinst', 'postinst-pycompile', '')
- dh.autoscript(package, 'prerm', 'prerm-pyclean', '')
- pyclean_added = True
-
- for pdir, details in stats['private_dirs'].iteritems():
- if not details.get('compile'):
- continue
- if not pyclean_added:
- dh.autoscript(package, 'prerm', 'prerm-pyclean', '')
- pyclean_added = True
-
- args = pdir
-
- ext_for = details.get('public_ext')
- if ext_for is None: # no extension
- if options.vrange:
- args += " -V %s" % vrange_str(options.vrange)
- elif False in ext_for: # extension's version not detected
- if options.vrange and '-' not in vrange_str(options.vrange):
- ver = vrange_str(options.vrange)
- else: # try shebang or default Python version
- ver = (list(v for i, v in details.get('shebangs', [])
- if v) or [None])[0] or DEFAULT
- args += " -V %s" % vrepr(ver)
- else:
- args += " -V %s" % vrepr(ext_for.pop())
-
- for pattern in options.regexpr or []:
- args += " -X '%s'" % pattern.replace("'", r"\'")
-
- dh.autoscript(package, 'postinst', 'postinst-pycompile', args)
-
- pydist_file = join('debian', "%s.pydist" % package)
- if exists(pydist_file):
- if not validate_pydist(pydist_file, True):
- log.warning("%s.pydist file is invalid", package)
- else:
- dstdir = join('debian', package, 'usr/share/python/dist/')
- if not exists(dstdir):
- os.makedirs(dstdir)
- fcopy(pydist_file, join(dstdir, package))
-
- dh.save()
-
- if __name__ == '__main__':
- main()
-